{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PII with regex and Presidio on 100 python samples, on source code and on extracted docstrings+comments "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "from bigscience_pii_detect_redact import run_pii_batch\n",
    "from functools import partial"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Email and IP address detection in the source code files with regex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_check_ds(ds, args):\n",
    "    ds_checks = ds.filter(\n",
    "        lambda exs: exs[\"modified\"],\n",
    "        batched=True,\n",
    "        batch_size=args.batch_size,\n",
    "        num_proc=args.num_proc\n",
    "    )\n",
    "    ds_checks = ds\n",
    "    #idx_samples = random.sample(range(len(ds_checks)), min(len(ds_checks), args.check_sampling_size))\n",
    "    # first modified 100 cells\n",
    "    idx_samples = [i for i in range(100)]\n",
    "    ds_checks = ds_checks.select(idx_samples)\n",
    "    return ds_checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration codeparrot--codeparrot-clean-valid-826c6fd8b27e5523\n",
      "Found cached dataset json (/Users/loubnabenallal/.cache/huggingface/datasets/codeparrot___json/codeparrot--codeparrot-clean-valid-826c6fd8b27e5523/0.0.0/e6070c77f18f01a5ad4551a8b7edfba20b8438b7cad4d94e6ad9378022ce4aab)\n"
     ]
    }
   ],
   "source": [
    "ds = load_dataset(\"codeparrot/codeparrot-clean-valid\", split=\"train\")\n",
    "ds = ds.select([i for i in range(100)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e9fe0d55f274583b8628da9fb13273a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#0:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89961b8cabba41d594b7352c3a47e2d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#1:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc1c9551048a4d65a8917deff4ae62bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#2:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d824c100b8f4ad6bc93a56344997d4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#3:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc3adad041354d648e9230fee4da0ed3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#4:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "243c5f60fedb46199d293965e1457d00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#5:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c229094b39344d01a39606ef86521b0e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#6:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "138e99566f0d4a319ea7c2c2c76c4e93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#8:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bb4925b0e8f47ed9866cf1626e3abe7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#9:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3726daca18e4dde9cbde07da88a40d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#7:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4aa7fda3a5ca45188b2459c829386442",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#10:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b21c7d28cd5540d989b8c80d4e352642",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#11:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds_pii = ds.map(\n",
    "    partial(run_pii_batch),\n",
    "    batched=True,\n",
    "    batch_size=10,\n",
    "    num_proc=12,\n",
    "    load_from_cache_file=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 17, 26, 29, 34, 35, 39, 45, 47, 48, 52, 57, 73, 76, 84, 88, 94, 96]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# indexes of files where PII was detected\n",
    "[i for i in range(100) if ds_pii[\"modified\"][i]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc10239e8445454995248b3b61ef3cd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#0:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7e554ef8fd34462ad9a3d3d752fef4f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#1:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "804df2c2b8914552a96ef5969e65c28d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#2:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e5891f1e02b49df85333a957d960391",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#3:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d251cf6bcf7408a96983dab55e9ee8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#4:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d92de148caa4c78a9464845c9c7b421",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#5:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc4de5c3f29f4020b97bdc2c3684c428",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#6:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58cc0543c7f84ebca497ab6802a1158c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#7:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efbaddbc01ab47908cba820ed1d8aa5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#8:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23341c3d688f4910acd4fcf0e5767bd2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#9:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40058f6ab7e34cfb89a9df6ddbfd7a5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#10:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "816b7dee560d4b66bf51467aa97ee2c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "#11:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds_modified = ds_pii.filter(\n",
    "    lambda exs: exs[\"modified\"],\n",
    "    batched=True,\n",
    "    batch_size=10,\n",
    "    num_proc=12,\n",
    "    load_from_cache_file=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['repo_name', 'path', 'copies', 'size', 'content', 'license', 'hash', 'line_mean', 'line_max', 'alpha_frac', 'autogenerated', 'regex_metadata', 'old_text', 'modified'],\n",
       "    num_rows: 18\n",
       "})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 18 files were modified\n",
    "ds_modified"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[(\\'127.0.0.1\\', (759, 770), \\'regex.Regex(\\\\\\'(?:^|[\\\\\\\\\\\\\\\\b\\\\\\\\\\\\\\\\s@?,!;:\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'\\\\\\\\\\\\\\\\\")(.\\\\\\\\\\\\\\\\p{Han}])((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\\\\\\\\\\\\\\\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\\\\\\\\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\\\\\\\\\.){3,3}(25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:$|[\\\\\\\\\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\"(.\\\\\\\\\\\\\\\\p{Han}])\\\\\\', flags=regex.M | regex.V0)\\', \\'IP_ADDRESS\\'), (\\'127.0.0.1\\', (8079, 8090), \\'regex.Regex(\\\\\\'(?:^|[\\\\\\\\\\\\\\\\b\\\\\\\\\\\\\\\\s@?,!;:\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'\\\\\\\\\\\\\\\\\")(.\\\\\\\\\\\\\\\\p{Han}])((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\\\\\\\\\\\\\\\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\\\\\\\\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\\\\\\\\\.){3,3}(25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:$|[\\\\\\\\\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\"(.\\\\\\\\\\\\\\\\p{Han}])\\\\\\', flags=regex.M | regex.V0)\\', \\'IP_ADDRESS\\')]'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ds_modified[\"regex_metadata\"][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[(\\'<cedric.bellegarde@adishatz.org>\\', (44, 76), \\'regex.Regex(\\\\\\'([^\\\\\\\\\\\\\\\\s@,?!;:)(]+@[^,\\\\\\\\\\\\\\\\s!?;,]{3,}[\\\\\\\\\\\\\\\\.][^\\\\\\\\\\\\\\\\s\\\\\\\\\\\\\\\\b\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'\\\\\\\\\\\\\\\\\"@,?!;:)(.]+)\\\\\\', flags=regex.M | regex.V0)\\', \\'EMAIL\\')]'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_modified[\"regex_metadata\"][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[(\\'ivana.mihalek@gmail.com\\', (717, 740), \\'regex.Regex(\\\\\\'([^\\\\\\\\\\\\\\\\s@,?!;:)(]+@[^,\\\\\\\\\\\\\\\\s!?;,]{3,}[\\\\\\\\\\\\\\\\.][^\\\\\\\\\\\\\\\\s\\\\\\\\\\\\\\\\b\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'\\\\\\\\\\\\\\\\\"@,?!;:)(.]+)\\\\\\', flags=regex.M | regex.V0)\\', \\'EMAIL\\')]'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_modified[\"regex_metadata\"][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[(\\'/build/debpkg/usr/local/lib/mxcart/my_tilers_tools/\\', (1829, 1881), \\'regex.Regex(\\\\\\'((?:(?:[A-Za-z]+[\\\\\\\\\\\\\\\\p{Nd}\\\\\\\\\\\\\\\\p{Pd}\\\\\\\\\\\\\\\\/\\\\\\\\\\\\\\\\+\\\\\\\\\\\\\\\\=:_]+|[\\\\\\\\\\\\\\\\p{Nd}\\\\\\\\\\\\\\\\p{Pd}\\\\\\\\\\\\\\\\/\\\\\\\\\\\\\\\\+\\\\\\\\\\\\\\\\=:]+[A-Za-z]+)){10,})(?:$|[\\\\\\\\\\\\\\\\b\\\\\\\\\\\\\\\\s\\\\\\\\\\\\\\\\p{Han}@?,!;:\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'\\\\\\\\\\\\\\\\\")(.])\\\\\\', flags=regex.M | regex.V0)\\', \\'KEY\\'), (\\'/build/debpkg/usr/local/share/icons/hicolor/48x48/apps/\\', (2814, 2870), \\'regex.Regex(\\\\\\'((?:(?:[A-Za-z]+[\\\\\\\\\\\\\\\\p{Nd}\\\\\\\\\\\\\\\\p{Pd}\\\\\\\\\\\\\\\\/\\\\\\\\\\\\\\\\+\\\\\\\\\\\\\\\\=:_]+|[\\\\\\\\\\\\\\\\p{Nd}\\\\\\\\\\\\\\\\p{Pd}\\\\\\\\\\\\\\\\/\\\\\\\\\\\\\\\\+\\\\\\\\\\\\\\\\=:]+[A-Za-z]+)){10,})(?:$|[\\\\\\\\\\\\\\\\b\\\\\\\\\\\\\\\\s\\\\\\\\\\\\\\\\p{Han}@?,!;:\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'\\\\\\\\\\\\\\\\\")(.])\\\\\\', flags=regex.M | regex.V0)\\', \\'KEY\\'), (\\'Icon=/usr/local/share/icons/hicolor/48x48/apps/mxcart\\', (3728, 3782), \\'regex.Regex(\\\\\\'((?:(?:[A-Za-z]+[\\\\\\\\\\\\\\\\p{Nd}\\\\\\\\\\\\\\\\p{Pd}\\\\\\\\\\\\\\\\/\\\\\\\\\\\\\\\\+\\\\\\\\\\\\\\\\=:_]+|[\\\\\\\\\\\\\\\\p{Nd}\\\\\\\\\\\\\\\\p{Pd}\\\\\\\\\\\\\\\\/\\\\\\\\\\\\\\\\+\\\\\\\\\\\\\\\\=:]+[A-Za-z]+)){10,})(?:$|[\\\\\\\\\\\\\\\\b\\\\\\\\\\\\\\\\s\\\\\\\\\\\\\\\\p{Han}@?,!;:\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'\\\\\\\\\\\\\\\\\")(.])\\\\\\', flags=regex.M | regex.V0)\\', \\'KEY\\')]'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_modified[\"regex_metadata\"][3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[(\\'andrew.t.bentley@gmail.com\\', (720, 746), \\'regex.Regex(\\\\\\'([^\\\\\\\\\\\\\\\\s@,?!;:\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'=)(]+@[^,\\\\\\\\\\\\\\\\s!?;,]{3,}[\\\\\\\\\\\\\\\\.][^\\\\\\\\\\\\\\\\s\\\\\\\\\\\\\\\\b\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'\\\\\\\\\\\\\\\\\"@,?!;:)(.]+)\\\\\\', flags=regex.M | regex.V0)\\', \\'EMAIL\\')]'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_modified[\"regex_metadata\"][4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sion(),\n",
      "    url='http://github.com/atbentley/plank/',\n",
      "    license='MIT',\n",
      "    author='Andrew Bentley',\n",
      "    author_email='andrew.t.bentley@gmail.com',\n",
      "    description=\"A simple task and build runner tha\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"old_text\"][4][600:800])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sion(),\n",
      "    url='http://github.com/atbentley/plank/',\n",
      "    license='MIT',\n",
      "    author='Andrew Bentley',\n",
      "    author_email='dummy@email.com',\n",
      "    description=\"A simple task and build runner that doesn't g\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"content\"][4][600:800])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('192.168.88.1', (1886, 1900), 'regex.Regex(\\'(?:^|[\\\\\\\\b\\\\\\\\s@?,!;:\\\\\\\\\\\\\\'\\\\\\\\\")(.\\\\\\\\p{Han}])((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\\\\\\\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:$|[\\\\\\\\s@,?!;:\\\\\\'\"(.\\\\\\\\p{Han}])\\', flags=regex.M | regex.V0)', 'IP_ADDRESS')]\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"regex_metadata\"][5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('9bea85795705d015cdadc82c68b99196a8554f5c', (34916, 34957), 'regex.Regex(\\'((?:(?:[A-Za-z]+[\\\\\\\\p{Nd}\\\\\\\\p{Pd}\\\\\\\\/\\\\\\\\+\\\\\\\\=:_]+|[\\\\\\\\p{Nd}\\\\\\\\p{Pd}\\\\\\\\/\\\\\\\\+\\\\\\\\=:]+[A-Za-z]+)){10,})(?:$|[\\\\\\\\b\\\\\\\\s\\\\\\\\p{Han}@?,!;:\\\\\\\\\\\\\\'\\\\\\\\\")(.])\\', flags=regex.M | regex.V0)', 'KEY'), ('super@example.com', (53204, 53221), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'=)(]+@[^,\\\\\\\\s!?;,]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('super@example.com', (53387, 53404), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'=)(]+@[^,\\\\\\\\s!?;,]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('super@example.com', (55475, 55492), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'=)(]+@[^,\\\\\\\\s!?;,]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('//input[@value=\"Save\"]\\').click', (175046, 175076), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'=)(]+@[^,\\\\\\\\s!?;,]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('//input[@value=\"Save\"]\\').click', (178159, 178189), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'=)(]+@[^,\\\\\\\\s!?;,]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL')]\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"regex_metadata\"][9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f.super_email_login = {\n",
      "            REDIRECT_FIELD_NAME: '/test_admin/admin/',\n",
      "            'username': 'super@example.com',\n",
      "            'password': 'secret',\n",
      "        }\n",
      "        self.super_email_bad_login = {\n",
      "            REDIRECT_FIELD_NAME: '/test_admin/admin/',\n",
      "            'username': 'super@example\n"
     ]
    }
   ],
   "source": [
    "#(ds_modified[\"old_text\"][9]).index(\"super@\")\n",
    "print(ds_modified[\"old_text\"][9][53100:53400])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('\"aeronaut@pianoguy.de', (319, 340), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'=)(]+@[^,\\\\\\\\s!?;,]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('\"vuolter@gmail.com', (386, 404), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'=)(]+@[^,\\\\\\\\s!?;,]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL')]\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"regex_metadata\"][6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('marc@marc-abramowitz.com', (249, 273), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'=)(]+@[^,\\\\\\\\s!?;,]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL')]\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"regex_metadata\"][7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('192.168.1.2', (809, 822), 'regex.Regex(\\'(?:^|[\\\\\\\\b\\\\\\\\s@?,!;:\\\\\\\\\\\\\\'\\\\\\\\\")(.\\\\\\\\p{Han}])((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\\\\\\\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:$|[\\\\\\\\s@,?!;:\\\\\\'\"(.\\\\\\\\p{Han}])\\', flags=regex.M | regex.V0)', 'IP_ADDRESS')]\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"regex_metadata\"][8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('10.254.254.28', (453, 468), 'regex.Regex(\\'(?:^|[\\\\\\\\b\\\\\\\\s@?,!;:\\\\\\\\\\\\\\'\\\\\\\\\")(.\\\\\\\\p{Han}])((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\\\\\\\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:$|[\\\\\\\\s@,?!;:\\\\\\'\"(.\\\\\\\\p{Han}])\\', flags=regex.M | regex.V0)', 'IP_ADDRESS')]\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"regex_metadata\"][11])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ort sys\n",
      "from urllib.request import urlretrieve\n",
      "from pathlib import Path\n",
      "\n",
      "\"\"\"Logpuzzle exercise\n",
      "Given an apache logfile, find the puzzle urls and download the images.\n",
      "\n",
      "Here's what a puzzle url looks like:\n",
      "10.254.254.28 - - [06/Aug/2007:00:13:48 -0700] \"GET /~foo/puzzle-bar-aaab.jpg HTTP/1.0\" 302 528 \"-\" \"Mozilla/5.0 (Wi\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"old_text\"][11][250:570])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('zoidberg@mujmail.cz', (390, 409), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL')]\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"regex_metadata\"][12])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('<wackou@gmail.com>', (140, 158), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('0.0.0.0', (14127, 14136), 'regex.Regex(\\'(?:^|[\\\\\\\\b\\\\\\\\s@?,!;:\\\\\\\\\\\\\\'\\\\\\\\\")(.\\\\\\\\p{Han}])((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\\\\\\\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:$|[\\\\\\\\s@,?!;:\\\\\\'\"(.\\\\\\\\p{Han}])\\', flags=regex.M | regex.V0)', 'IP_ADDRESS'), ('127.0.0.1', (18960, 18971), 'regex.Regex(\\'(?:^|[\\\\\\\\b\\\\\\\\s@?,!;:\\\\\\\\\\\\\\'\\\\\\\\\")(.\\\\\\\\p{Han}])((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\\\\\\\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:$|[\\\\\\\\s@,?!;:\\\\\\'\"(.\\\\\\\\p{Han}])\\', flags=regex.M | regex.V0)', 'IP_ADDRESS'), ('123.123.123.123', (23248, 23265), 'regex.Regex(\\'(?:^|[\\\\\\\\b\\\\\\\\s@?,!;:\\\\\\\\\\\\\\'\\\\\\\\\")(.\\\\\\\\p{Han}])((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\\\\\\\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\\\\\\\.){3,3}(25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))(?:$|[\\\\\\\\s@,?!;:\\\\\\'\"(.\\\\\\\\p{Han}])\\', flags=regex.M | regex.V0)', 'IP_ADDRESS')]\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"regex_metadata\"][13])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23249"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ds_modified[\"old_text\"][13]).index(\"123.123.123.123\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ironment(args.environment)\n",
      "        print()\n",
      "        if len(args.args) != 2:\n",
      "            log.error('You need to specify a deployment config file as argument and a host ip or vps provider')\n",
      "            log.error('eg: bts deploy_node deploy_config.yaml 123.123.123.123  # use given host for install')\n",
      "   \n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"old_text\"][13][23000:23300])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('ngakan.gandhi@packet-systems.com', (53, 85), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL')]\n",
      "[('<amix.pal@gmail.com>', (114, 134), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL')]\n",
      "[('<sduenas@bitergia.com>', (730, 752), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('<quan@bitergia.com>', (769, 788), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('P<email>[^\\\\s@]+@[^\\\\s@.]+\\\\.[^\\\\s', (2365, 2395), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('jsmith@example.com', (6961, 6979), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('jsmith@example.net', (6983, 7001), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('jsmith@example.net', (7010, 7028), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('johnsmith@example.com', (7032, 7053), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('jdoe@example.com', (7062, 7078), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('john_doe@example.com', (7084, 7104), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('john_doe@example.com', (7135, 7155), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('jsmith@example.com', (7941, 7959), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('jdoe@example.com', (8000, 8016), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('jsmith@example.com', (8057, 8075), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL')]\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"regex_metadata\"][14])\n",
    "print(ds_modified[\"regex_metadata\"][15])\n",
    "print(ds_modified[\"regex_metadata\"][16])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('<niphlod@gmail.com>', (49, 68), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL'), ('<niphlod@gmail.com>', (983, 1002), 'regex.Regex(\\'([^\\\\\\\\s@,?!;:\\\\\\\\\\\\\\'\\\\\\\\\"=)(]+@[^,\\\\\\\\s!?;,\\\\\\\\\\\\\\'\\\\\\\\\"=]{3,}[\\\\\\\\.][^\\\\\\\\s\\\\\\\\b\\\\\\\\\\\\\\'\\\\\\\\\"@,?!;:)(.]+)\\', flags=regex.M | regex.V0)', 'EMAIL')]\n"
     ]
    }
   ],
   "source": [
    "print(ds_modified[\"regex_metadata\"][17])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Presidio (Person names detection in the docstrings/comments)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this section we test Presidio for the detection of the entities [\"EMAIL_ADDRESS\",\"PERSON\", \"CREDIT_CARD\", \"IP_ADDRESS\", \"IBAN_CODE\"]\n",
    "* We first apply it on the code files then on their dosctrings/comments only\n",
    "\n",
    "* Even with a high threshold on prediction score (0.8) the **predictiosn are bad**. The underhood NER must have trouble infering context from technical text\n",
    "which often include python keywords and code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install presidio-analyzer\n",
    "!pip install presidio-anonymizer\n",
    "!python -m spacy download en_core_web_lg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "import pandas as pd\n",
    "from text_extraction import get_text\n",
    "from presidio_analyzer import AnalyzerEngine, BatchAnalyzerEngine\n",
    "\n",
    "COLUMN=\"content\"\n",
    " \n",
    "def extract_nl_text(example):\n",
    "        text = get_text(example[COLUMN])\n",
    "        example[\"nl_text\"] = text\n",
    "        example[\"nl_size\"] = len(text)\n",
    "        return example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class docstring\n",
      "function docstring\n",
      "@since Jan 6, 2011\n",
      "@author Mario Steinhoff\n",
      "\n",
      "first comment second comment\n"
     ]
    }
   ],
   "source": [
    "example = '''\n",
    "\"\"\"\n",
    "@since Jan 6, 2011\n",
    "@author Mario Steinhoff\n",
    "\"\"\"\n",
    "\n",
    "class foo():\n",
    "    \"\"\"\n",
    "    class docstring\n",
    "    \"\"\"\n",
    "    some_value = 2\n",
    "    def bar(arg1, arg2):\n",
    "        \"\"\" function docstring\"\"\"\n",
    "        if arg1 == 2:\n",
    "            #first comment\n",
    "            print(arg1)\n",
    "        else:\n",
    "            # second comment\n",
    "            arg2 += 3\n",
    "        return arg1 + arg2\n",
    "'''\n",
    "\n",
    "print(get_text(example))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration codeparrot--codeparrot-clean-valid-826c6fd8b27e5523\n",
      "Found cached dataset json (/Users/loubnabenallal/.cache/huggingface/datasets/codeparrot___json/codeparrot--codeparrot-clean-valid-826c6fd8b27e5523/0.0.0/e6070c77f18f01a5ad4551a8b7edfba20b8438b7cad4d94e6ad9378022ce4aab)\n",
      "Loading cached processed dataset at /Users/loubnabenallal/.cache/huggingface/datasets/codeparrot___json/codeparrot--codeparrot-clean-valid-826c6fd8b27e5523/0.0.0/e6070c77f18f01a5ad4551a8b7edfba20b8438b7cad4d94e6ad9378022ce4aab/cache-705f9821443934eb.arrow\n"
     ]
    }
   ],
   "source": [
    "# dataset for pII\n",
    "ds = load_dataset(\"codeparrot/codeparrot-clean-valid\", split=\"train\")\n",
    "ds = ds.select([i for i in range(100)])\n",
    "ds_text = ds.map(extract_nl_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['repo_name', 'path', 'copies', 'size', 'content', 'license', 'hash', 'line_mean', 'line_max', 'alpha_frac', 'autogenerated', 'nl_text', 'nl_size'],\n",
       "    num_rows: 100\n",
       "})"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "code_content = ds_text[\"content\"]\n",
    "code_content_df = pd.DataFrame(code_content, columns=[\"content\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analyzing 100 code snippets\n"
     ]
    }
   ],
   "source": [
    "# Apply Presidio\n",
    "print(f\"Analyzing {len(code_content_df)} code snippets\")\n",
    "analyzer = AnalyzerEngine()\n",
    "# Add regexes to analyzer here\n",
    "batch_analyzer = BatchAnalyzerEngine(analyzer_engine=analyzer)\n",
    "analyzer_results = batch_analyzer.analyze_dict(\n",
    "    code_content_df.to_dict(orient=\"list\"),\n",
    "    language=\"en\",\n",
    "    entities=[\"EMAIL_ADDRESS\",\"PERSON\", \"CREDIT_CARD\", \"IP_ADDRESS\", \"IBAN_CODE\"],\n",
    "    score_threshold=0.8\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "detected_entities_list = []\n",
    "for result in analyzer_results:\n",
    "    recognizer_results = result.recognizer_results\n",
    "    content = result.value\n",
    "    for recognizer_result, code in zip(recognizer_results, content):\n",
    "        entities = []\n",
    "        for entity in recognizer_result:\n",
    "            entities.append([entity.entity_type, code[entity.start : entity.end], entity.score])\n",
    "        current_code_entities = [code, len(code), entities]\n",
    "        detected_entities_list.append(current_code_entities)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualization of some examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample 0:\n",
      "[['PERSON', 'DATASET_VIEWS', 0.85], ['PERSON', 'pgresults', 0.85], ['PERSON', 'pgresults', 0.85], ['PERSON', 'has_write_permissions', 0.85]]\n",
      "\n",
      "Sample 1:\n",
      "[['PERSON', 'Mikio Hirabayashi', 0.85], ['PERSON', \"outlst = [struct.pack('>BBI'\", 0.85], ['PERSON', 'fail_code = ord(sockrecv(sock', 0.85], ['PERSON', 'klen', 0.85], ['PERSON', 'maxkeys = len(self', 0.85], ['PERSON', 'self.sock.close', 0.85], ['PERSON', 'bitflag', 0.85], ['PERSON', 't0(C.rnum', 0.85], ['PERSON', 'tcrdbmisc', 0.85], ['PERSON', 'bitflag', 0.85]]\n",
      "\n",
      "Sample 2:\n",
      "[['PERSON', 'mignight', 0.85]]\n",
      "\n",
      "Sample 3:\n",
      "[['PERSON', 'TraxelsPerFrame[0', 0.85], ['PERSON', 'hypotheses_graph.toTrackingGraph', 0.85]]\n",
      "\n",
      "Sample 4:\n",
      "[['PERSON', '轮廓提取模式 Contour_Retrieval_Mode\\nimage', 0.85], ['PERSON', 'img', 0.85]]\n",
      "\n",
      "Sample 5:\n",
      "[['PERSON', \"uses_free\\n\\n\\n@pytest.fixture(scope='module'\", 0.85], ['PERSON', 'start_line = test_lnotab_roundtrip.__code__.co_firstlineno + 3', 0.85], ['PERSON', 'CO_FUTURE_ABSOLUTE_IMPORT=True', 0.85], ['PERSON', 'CO_FUTURE_WITH_STATEMENT=True', 0.85], ['PERSON', 'Flag.max', 0.85], ['PERSON', 'uses_free', 0.85]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(6):\n",
    "    print(f\"Sample {i}:\\n{detected_entities_list[i][-1]}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Apply Presidio on docstrings and comments (mostly english text)\n",
    "\n",
    "Presidio doesn't work on source code, let's try on docstrings and comments which should mostly be englih text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Displays a MX Dataset and associated information.\\n\\nShows a full (hundreds of images) dataset its metadata and a list\\nof associated files with the option to show metadata of each file\\nand ways to download those files.  With write permission this page\\nalso allows uploading and metadata editing.\\n\\nSettings for this view:\\nINSTALLED_APPS += (\"tardis.apps.mx_views\",)\\nDATASET_VIEWS = [(\"http://synchrotron.org.au/views/dataset/full\",\\n                  \"tardis.apps.mx_views.views.view_full_dataset\"),]\\n\\n pagination was removed by someone in the interface but not here. need to fix. If page request (9999) is out of range, deliver last page of results. take 4 evenly spaced images from the set'"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#example of extracted docstrings + comments\n",
    "ds_text[\"nl_text\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "#select extracted text\n",
    "code_content = ds_text[\"nl_text\"]\n",
    "code_content_df = pd.DataFrame(code_content, columns=[\"nl_text\"])\n",
    "\n",
    "analyzer = AnalyzerEngine()\n",
    "batch_analyzer = BatchAnalyzerEngine(analyzer_engine=analyzer)\n",
    "analyzer_results = batch_analyzer.analyze_dict(\n",
    "    code_content_df.to_dict(orient=\"list\"),\n",
    "    language=\"en\",\n",
    "    entities=[\"EMAIL_ADDRESS\",\"PERSON\", \"CREDIT_CARD\", \"IP_ADDRESS\", \"IBAN_CODE\"],\n",
    "    score_threshold=0.8\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "detected_entities_list2 = []\n",
    "for result in analyzer_results:\n",
    "    recognizer_results = result.recognizer_results\n",
    "    content = result.value\n",
    "    for recognizer_result, code in zip(recognizer_results, content):\n",
    "        entities = []\n",
    "        for entity in recognizer_result:\n",
    "            if entity.score >= 0.8:\n",
    "                entities.append([entity.entity_type, code[entity.start : entity.end], entity.score])\n",
    "        current_code_entities = [code, len(code), entities]\n",
    "        detected_entities_list2.append(current_code_entities)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample 0:\n",
      "[['PERSON', 'DATASET_VIEWS', 0.85]]\n",
      "\n",
      "Sample 1:\n",
      "[['PERSON', 'bitflag', 0.85], ['PERSON', 'bitflag', 0.85], ['PERSON', 'Mikio Hirabayashi', 0.85]]\n",
      "\n",
      "Sample 2:\n",
      "[['PERSON', 'mignight', 0.85]]\n",
      "\n",
      "Original text 2:\n",
      "wrong precision wrong prefix yesterday is always save since we reset to mignight and do <, not <=\n",
      "\n",
      "Sample 3:\n",
      "[['PERSON', 'TraxelsPerFrame[0', 0.85], ['PERSON', 'trackingGraph', 0.85], ['PERSON', 'hypotheses_graph.toTrackingGraph', 0.85]]\n",
      "\n",
      "Sample 4:\n",
      "[['PERSON', 'type轮廓提取模式 Contour_Retrieval_Mode img = cv2.drawContours(im', 0.85]]\n",
      "\n",
      "Sample 8:\n",
      "[['PERSON', 'Luc Saffre', 0.85]]\n",
      "\n",
      "Sample 9:\n",
      "[['PERSON', 'TODO', 0.85], ['PERSON', 'TODO', 0.85], ['PERSON', 'TODO', 0.85]]\n",
      "\n",
      "Sample 11:\n",
      "[['PERSON', 'utf-8============================================================================================================================================================================================html_doc', 0.85]]\n",
      "\n",
      "Sample 14:\n",
      "[['PERSON', 'Creer', 0.85], ['PERSON', 'Creer', 0.85], ['PERSON', 'Creer', 0.85], ['PERSON', 'Creer', 0.85], ['PERSON', 'Creer', 0.85]]\n",
      "\n",
      "Sample 15:\n",
      "[['PERSON', 'TODO', 0.85], ['PERSON', 'TODO', 0.85], ['PERSON', 'TODO', 0.85], ['PERSON', 'TODO', 0.85], ['PERSON', 'TODO', 0.85]]\n",
      "\n",
      "Sample 17:\n",
      "[['EMAIL_ADDRESS', 'cedric.bellegarde@adishatz.org', 1.0], ['PERSON', 'Cedric Bellegarde', 0.85]]\n",
      "\n",
      "Sample 19:\n",
      "[['PERSON', 'Meresco', 0.85], ['PERSON', 'Meresco', 0.85], ['PERSON', 'Meresco', 0.85], ['PERSON', 'Meresco', 0.85], ['PERSON', 'Meresco', 0.85], ['PERSON', 'Meresco', 0.85], ['PERSON', 'Franklin St', 0.85]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(20):\n",
    "    if detected_entities_list2[i][-1]:\n",
    "        print(f\"Sample {i}:\\n{detected_entities_list2[i][-1]}\\n\")\n",
    "    if i == 2:\n",
    "        print(f\"Original text {i}:\\n{detected_entities_list2[i][0]}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['PERSON', 'Meresco', 0.85],\n",
       " ['PERSON', 'Meresco', 0.85],\n",
       " ['PERSON', 'Meresco', 0.85],\n",
       " ['PERSON', 'Meresco', 0.85],\n",
       " ['PERSON', 'Meresco', 0.85],\n",
       " ['PERSON', 'Meresco', 0.85],\n",
       " ['PERSON', 'Franklin St', 0.85]]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detected_entities_list2[19][-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "begin license   \"Meresco Examples\" is a project demonstrating some of the features of various components of the \"Meresco Suite\". Also see http://meresco.org.   Copyright (C) 2007-2008 SURF Foundation. http://www.surf.nl Copyright (C) 2007-2010 Seek You Too (CQ2) http://www.cq2.nl Copyright (C) 2007-2009 Stichting Kennisnet Ict op school. http://www.kennisnetictopschool.nl Copyright (C) 2009 Delft University of Technology http://www.tudelft.nl Copyright (C) 2009 Tilburg University http://www.uvt.nl  This file is part of \"Meresco Examples\"  \"Meresco Examples\" is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.  \"Meresco Examples\" is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.  You should have received a copy of the GNU General Public License along with \"Meresco Examples\"; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA  end license\n"
     ]
    }
   ],
   "source": [
    "print(detected_entities_list2[19][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['PERSON', 'Args', 0.85], ['PERSON', 'Args', 0.85]]\n"
     ]
    }
   ],
   "source": [
    "print(detected_entities_list2[20][-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['PERSON', 'MAASAPINotFound', 0.85], ['PERSON', 'ExternalAuthInfo', 0.85], ['PERSON', 'json', 0.85], ['PERSON', 'rndc', 0.85]]\n",
      "[['PERSON', 'ExternalComponentsMiddleware', 0.85], ['PERSON', 'MAASAPINotFound', 0.85], ['PERSON', 'json', 0.85], ['PERSON', 'rndc', 0.85], ['PERSON', 'INTERNAL_SERVER_ERROR', 0.85], ['PERSON', 'logger.error', 0.85], ['PERSON', 'GATEWAY_TIMEOUT', 0.85]]\n"
     ]
    }
   ],
   "source": [
    "print(detected_entities_list2[25][-1])\n",
    "print(detected_entities_list[25][-1])"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "fd8fde6f83dada9276d12fdb71d773558994168ed1b3bea457b8db38c02aa2e1"
  },
  "kernelspec": {
   "display_name": "Python 3.10.4 ('venv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
